#include "stdafx.h"
#include <spinn3r.h>
#include <memes.h>

void BigMain(int argc, char* argv[]);
void QuotesForDepression();

int main(int argc, char* argv[]) {
  printf("MemeWorks. build: %s, %s. Start time: %s\n\n", __TIME__, __DATE__, TExeTm::GetCurTm());
  //TSysProc::SetLowPriority();
  TExeTm ExeTm;  TInt::Rnd.PutSeed(0);  Try
  //BigMain(argc, argv); return 0;
  char *ToDo [] = {"QuotesApp", "-do:mkdataset", "-i:_files.txt"};  BigMain(3, ToDo);  return 0;
  //char *ToDo [] = {"QuotesApp", "-do:memestoqtbs", "-i:f.txt"};  BigMain(3, ToDo);  return 0;

  //////
  TIntPrV KeeepEdgeV;
    
  //TIntV MnWV = TIntV::GetV(5,6,7,8), MxWV = TIntV::GetV(10,20,30), MnFqV = TIntV::GetV(10,20,30);
  //for (int v1 = 0; v1 < MnWV.Len(); v1++) {
  //for (int v2 = 0; v2 < MxWV.Len(); v2++) {
  //for (int v3 = 0; v3 < MnFqV.Len(); v3++) {
  /*  PQuoteBs QtBs = TQuoteBs::Load(TZipIn("qt3-080901-090831-w4mfq10.QtBs.rar"));
    PClustNet Net = TClustNet::Load(TZipIn("qt3-080901-090831-w4fq10.ClustNet.rar"));
    TSnap::PrintInfo(Net);
    QtBs->Dump(false);
    return 0;
    Net->RecalcEdges(0.8);
    TSnap::PrintInfo(Net);
    printf("get subgraph\n");
    //const int MnW = MnWV[v1], MxW = MxWV[v2], MnFq = MnFqV[v3];
    int MnW =5, MxW=30, MnFq=10;
    TStr Pref = TStr::Fmt("qt3E08-w%d-%dfq%d", MnW, MxW, MnFq), Suff;
    Net = Net = Net->GetSubGraph(MnW, MxW, MnFq);  TSnap::PrintInfo(Net);
    printf("del self edges\n");
    TSnap::DelSelfEdges(Net);
    printf("find clusters\n");
    //Net->ClustGreedyTopDown(KeeepEdgeV); Suff="-TD";
    //Net->ClustKeepSingleEdge(1, KeeepEdgeV); Suff="-FQ";
    Net->ClustKeepSingleEdge(2, KeeepEdgeV);  Suff="-LG";
    //Net->ClustKeepSingleEdge(3, KeeepEdgeV); Suff="-SH";
    printf("make clusters\n");
    Net->MakeClusters(KeeepEdgeV);
    TVec<TIntV> ClustV, ClustV2;
    Net->DumpClustersByVol(TStr::Fmt("%s-w%d-%dfq%d%s", 
      Pref.CStr(), MnW, MxW, MnFq, Suff.CStr()), 5, 100);
  //} } }
  Net->GetClusters(ClustV);
  printf("Skipping cluster of %d quotes\n", ClustV[0].Len());
  TIntV QtIdV;
  for (int i = 1; i < ClustV.Len(); i++) {
    if (ClustV[i].Len() >= 3) {
      ClustV2.Add(ClustV[i]);
      QtIdV.AddV(ClustV[i]);
    }
  }
  printf("Get small QtBs\n");
  PQuoteBs SmallQtBs = QtBs->GetQuoteBs(QtIdV);
  printf("Create clusters\n");
  SmallQtBs->CreateClusters(ClustV2); // set clusters
  printf("Save\n");
  { TFOut FOut(Pref+"-clust5.QtBs"); SmallQtBs->Save(FOut); }
  //PQuoteBs SmallQtBs = TQuoteBs::Load(TZipIn("qt3-w7-20fq20-clust3.QtBs.gz"));
  SmallQtBs->SaveClusters(Pref+Suff+"1", true);
  SmallQtBs->SaveClusters(Pref+Suff+"2", false); // */

  PQuoteBs QtBs = TQuoteBs::Load(TZipIn("qt3E08-w5-30fq10-clust5.QtBs.gz"));
  QtBs->SaveClusters("qt3E08-w5-30fq10-clust5a", true);
  QtBs->SaveClusters("qt3E08-w5-30fq10-clust5b", false); 
  
  CatchFull
  printf("\nrun time: %s (%s)\n", ExeTm.GetTmStr(), TSecTm::GetCurTm().GetTmStr().CStr());
  return 0;
}

void BigMain(int argc, char* argv[]) {
  TExeTm ExeTm;
  Env = TEnv(argc, argv, TNotify::StdNotify);
  Env.PrepArgs("QuotesApp");
  const TStr ToDo = Env.GetIfArgPrefixStr("-do:", "", "To do").GetLc();
  if (Env.IsEndOfRun()) {
    printf("To do:\n");
    printf("    MkDataset         : Make memes dataset (extract quotes and save txt)\n");
    printf("    ExtractSubset     : Extract a subset of memes containing particular words\n");
    printf("    MemesToQtBs       : Load memes dataset and create quote base\n");
    printf("    MkClustNet        : Build cluster network from the quote base\n");
    return;
  }	
  // extract quotes and links and make them into a single file
  if (ToDo == "mkdataset") {
    const TStr InFNm = Env.GetIfArgPrefixStr("-i:", "files.txt", "Spinn3r input files (one file per line)");
    const TStr OutFNm = Env.GetIfArgPrefixStr("-o:", "Spinn3r-dataset.txt", "Output file");
    const int MinQtWrdLen = Env.GetIfArgPrefixInt("-w:", 3, "Minimum quote word length");
    const TStr UrlFNm = Env.GetIfArgPrefixStr("-u:", "", "Seen url set (THashSet<TMd5Sig>) file name");
    const bool UrlOnlyOnce = Env.GetIfArgPrefixBool("-q:", true, "Only keep unique Urls");
    //// parse directly from Spinn3r
    TStr Spinn3rFNm;
    THashSet<TMd5Sig> SeenUrlSet;
    if (UrlOnlyOnce && ! UrlFNm.Empty()) {  // keep track of already seen urls (so that there are no duplicate urls)
      TFIn FIn(UrlFNm);  SeenUrlSet.Load(FIn);
    }
    FILE *F = fopen(OutFNm.CStr(), "wt");
    TFIn FIn(InFNm);
    int Items=0;
    for (int f=0; FIn.GetNextLn(Spinn3rFNm); f++) {
      TQuoteExtractor QE(Spinn3rFNm.ToTrunc());
      printf("Processing %02d: %s [%s]\n", f+1, Spinn3rFNm.CStr(), TExeTm::GetCurTm());
      fflush(stdout);
      for (int item = 0; QE.Next(); item++) {
        const TMd5Sig PostMd5(QE.PostUrlStr);
        if (QE.QuoteV.Empty() && QE.LinkV.Empty()) { continue; } // no quotes, no links
        if (UrlOnlyOnce) {
          if (SeenUrlSet.IsKey(PostMd5)) { continue; }
          SeenUrlSet.AddKey(PostMd5);
        }
        fprintf(F, "P\t%s\n", QE.PostUrlStr.CStr());
        //if (QE.PubTm > TSecTm(2008,8,30) || QE.PubTm < TSecTm(2008,7,25)) { printf("%s\n", QE.PubTm.GetStr().CStr()); }
        fprintf(F, "T\t%s\n", QE.PubTm.GetYmdTmStr().CStr());
        for (int q = 0; q < QE.QuoteV.Len(); q++) {
          if (TStrUtil::CountWords(QE.QuoteV[q]) >= MinQtWrdLen) {
            fprintf(F, "Q\t%s\n", QE.QuoteV[q].CStr()); }
        }
        for (int l = 0; l < QE.LinkV.Len(); l++) {
          fprintf(F, "L\t%s\n", QE.LinkV[l].CStr()); }
        fprintf(F, "\n");
        if (item>0 && item % Kilo(100) == 0) {
          QE.DumpStat();  QE.ExeTm.Tick(); }
        Items++;
      }
      printf("file done. Total %d all posts, %d all items\n", SeenUrlSet.Len(), Items);
      fflush(stdout);
    }
    printf("all done. Saving %d post urls\n", SeenUrlSet.Len());  fflush(stdout);
    if (! SeenUrlSet.Empty()) {
      TFOut FOut(OutFNm.GetFMid()+".SeenUrlSet");
      SeenUrlSet.Save(FOut);
    }
    fclose(F);
  }
  // save posts with memes containing particular words
  else if (ToDo == "extractsubset") {
    const TStr InFNmWc = Env.GetIfArgPrefixStr("-i:", "memes_*.rar", "Input file prefix");
    const bool IsInFNmWc = Env.GetIfArgPrefixBool("-w:", true, "Input is wildcard (else a file with list of input files)");
    const TStr OutFNm = Env.GetIfArgPrefixStr("-o:", "memes-subset.txt", "Output memes file");
    const TStr WordsFNm = Env.GetIfArgPrefixStr("-p:", "phrases-in.txt", "Phrases that memes have to contain");

    TChAV CatchMemeV;// = TStr::GetV("great depression", "economic meltdown", "recession had bottomed out", "green shoots", "slow recovery", "gradual recovery");
    printf("Loading %s\n", WordsFNm.CStr());
    { TFIn FIn(WordsFNm);
    for (TStr Ln; FIn.GetNextLn(Ln); ) {
      printf("  %s\n", Ln.GetLc().CStr());
      CatchMemeV.Add(Ln.GetLc()); }
    }
    printf("%d strings loaded\n", CatchMemeV.Len());
    TFOut FOut(OutFNm);
    TMemesDataLoader Memes(InFNmWc, IsInFNmWc);
    for (int posts = 0, nsave=0; Memes.LoadNext(); posts++) {
      bool DoSave = false;
      for (int m = 0; m < Memes.MemeV.Len(); m++) {
        for (int i = 0; i < CatchMemeV.Len(); i++) {
          if (Memes.MemeV[m].SearchStr(CatchMemeV[i]) != -1) {
            DoSave=true; break; }
        }
        if (DoSave) { break; }
      }
      if (DoSave) { Memes.SaveTxt(FOut); nsave++; }
      if (posts % Mega(1) == 0) {
        printf("%dm posts, %d saved\n", posts/Mega(1), nsave);
        FOut.Flush();
      }
    }
  }
  // load memes dataset (MkDataset) and create quote base
  else if (ToDo == "memestoqtbs") {
    const TStr InFNmWc = Env.GetIfArgPrefixStr("-i:", "memes_200*.txt.rar", "Input Memes dataset files");
    const bool IsInFNmWc = Env.GetIfArgPrefixBool("-w:", true, "Input is wildcard (else a file with list of input files)");
    const TStr BlackListFNm = Env.GetIfArgPrefixStr("-b:", "quote_blacklist.txt", "Quote black list file");
    const TStr MediaUrlFNm = Env.GetIfArgPrefixStr("-u:", "news_media.txt", "Fule with news media urls");
    const TStr FNmPref = Env.GetIfArgPrefixStr("-o:", "qt08080902", "Output quote base");
    const int MinWrdLen = Env.GetIfArgPrefixInt("-l:", 4, "Min quote word length");
    const int MinMemeFq = Env.GetIfArgPrefixInt("-f:", 5, "Min meme frequency");
    THashSet<TChA> FqMemeSet, BlackList;
    TChA Ln;
    if (TFile::Exists(BlackListFNm)) {
      for (TFIn FIn(BlackListFNm); FIn.GetNextLn(Ln); ) {
        BlackList.AddKey(Ln); }
    }
    printf("Black list: %d\n", BlackList.Len());
    int NSkip=0;
    { TStrHash<TInt, TBigStrPool> MemeCntH(Mega(5));
    THashSet<TMd5Sig> SeenUrlSet;
    for (TMemesDataLoader Memes(InFNmWc, IsInFNmWc); Memes.LoadNext(); ) {
      TMd5Sig UrlSig(Memes.PostUrlStr);
      if (SeenUrlSet.IsKey(UrlSig)) { NSkip++; continue; }
      SeenUrlSet.AddKey(UrlSig);
      for (int m = 0; m < Memes.MemeV.Len(); m++) {
        MemeCntH.AddDat(Memes.MemeV[m].CStr()) += 1;
      }
    }
    printf("done [%s]\nAll quotes: %d\n", ExeTm.GetStr(), MemeCntH.Len());
    printf("  skip %d urls, keep %d\n", NSkip, SeenUrlSet.Len());
    { TFOut FOut(FNmPref+"-MemeFq.strHash");
      MemeCntH.Save(FOut); }
    for (int i = 0; i < MemeCntH.Len(); i++) {
      if (MemeCntH[i] >= 5 && ! BlackList.IsKey(MemeCntH.GetKey(i))) {
        FqMemeSet.AddKey(MemeCntH.GetKey(i)); }
    }
    SeenUrlSet.Clr(true); }
    printf("Frequent quotes: %d\n", FqMemeSet.Len());
    PQuoteBs QtBs = TQuoteBs::New();
    { THashSet<TMd5Sig> SeenUrlSet2;
    NSkip = 0;
    for (TMemesDataLoader Memes(InFNmWc, IsInFNmWc); Memes.LoadNext(); ) {
      TMd5Sig UrlSig(Memes.PostUrlStr);
      if (SeenUrlSet2.IsKey(UrlSig)) { NSkip++; continue; }
      SeenUrlSet2.AddKey(UrlSig);
      for (int m = 0; m < Memes.MemeV.Len(); m++) { // delete non-frequent memes
        if (! FqMemeSet.IsKey(Memes.MemeV[m])) {
          Memes.MemeV[m].Clr(); }
      }
      QtBs->AddQuote(Memes.MemeV, Memes.LinkV, Memes.PostUrlStr, Memes.PubTm, MinWrdLen);
    }
    printf("  skip %d urls, keep %d, add posts %d\n", NSkip, SeenUrlSet2.Len(), QtBs->Len()); }
    printf("SAVE 1: %d quotes\n", QtBs->Len());
    { TFOut FOut(TStr::Fmt("%s-w%dmfq%d.QtBs", FNmPref.CStr(), MinWrdLen, MinMemeFq)); QtBs->Save(FOut); }
    QtBs->SetUrlTy(MediaUrlFNm, utMedia);  // set url type
    printf("SAVE 2: %d quotes\n", QtBs->Len());
    { TFOut FOut(TStr::Fmt("%s-w%dmfq%d.QtBs", FNmPref.CStr(), MinWrdLen, MinMemeFq)); QtBs->Save(FOut); }
  }
  // make cluster network
  else if (ToDo == "mkclustnet") {
    const TStr InFNm = Env.GetIfArgPrefixStr("-i:", "QtBase-qt0808-0902w3mfq5\\qt08080902w3mfq5-clust.QtBs.rar", "Input quote base");
    const TStr OutFNm = Env.GetIfArgPrefixStr("-o:", "qt08080902w3mfq5", "Output network/updated QtBs filename");
    const int MinWrdLen = Env.GetIfArgPrefixInt("-l:", 4, "Min quote word length");
    const int MinMemeFq = Env.GetIfArgPrefixInt("-f:", 5, "Min meme frequency");
    const int ClustMeth = Env.GetIfArgPrefixInt("-c:", 1, "Clustering: keep edge to 1:most freq, 2: longest, 3:shortest, 4:random, 5:greedy top down");
    const TStr Pref = TStr::Fmt("%s-w%dfq%d", OutFNm.CStr(), MinWrdLen, MinMemeFq);
    PQuoteBs QtBs;
    if (TZipIn::IsZipFNm(InFNm)) { TZipIn ZipIn(InFNm);  QtBs = TQuoteBs::Load(ZipIn); }
    else { TFIn FIn(InFNm);  QtBs = TQuoteBs::Load(FIn); }
    PClustNet Net = TClustNet::GetFromQtBs(QtBs, MinMemeFq, MinWrdLen);
    TSnap::PrintInfo(Net);
    { printf("save clust net\n"); TFOut FOut(Pref+".ClustNet"); Net->Save(FOut); }
    // cluster
    TSnap::DelSelfEdges(Net);
    TIntPrV KeeepEdgeV;
    printf("cluster\n");
    if (ClustMeth == 5) { Net->ClustGreedyTopDown(KeeepEdgeV); }
    else { Net->ClustKeepSingleEdge(ClustMeth, KeeepEdgeV); }
    Net->MakeClusters(KeeepEdgeV);
    TVec<TIntV> ClustV;
    printf("get clusters\n");
    Net->GetClusters(ClustV);
    printf("set cluster\n");
    QtBs->CreateClusters(ClustV); // set clusters
    printf("save\n");
    { TFOut FOut(Pref+"-clust.QtBs"); QtBs->Save(FOut); }
    // save clusters
    TIntV QtIdV;
    QtBs->GetQtIdVByFq(QtIdV, MinWrdLen, 50, false); QtIdV.Del(100, QtIdV.Len()-1);
    QtBs->SaveQuotes(QtIdV, Pref+"-quotes.txt");
    QtBs->SaveClusters(Pref+"a", true);
    QtBs->SaveClusters(Pref+"b", false);
    const TUrlTy CntUrlTy = utUndef;
    TSecTm MinTm(2009,2,1), MaxTm(2009,7,31); //QtBs->GetMinMaxTm(MinTm, MaxTm);
    QtBs->GetQtIdVByFq(QtIdV, MinWrdLen, MinMemeFq, true, "", "", CntUrlTy);
    QtBs->SaveForFlash(QtIdV, Pref+"-top10", tmu4Hour, 10, MinTm, MaxTm, CntUrlTy);
    QtBs->SaveForFlash(QtIdV, Pref+"-top20", tmu4Hour, 20, MinTm, MaxTm, CntUrlTy);
    QtBs->SaveForFlash(QtIdV, Pref+"-top30", tmu4Hour, 30, MinTm, MaxTm, CntUrlTy);
    QtBs->SaveForFlash(QtIdV, Pref+"-top50", tmu4Hour, 50, MinTm, MaxTm, CntUrlTy);
    QtBs->SaveForFlash(QtIdV, Pref+"-top100", tmu4Hour, 100, MinTm, MaxTm, CntUrlTy);
  }
}

void QuotesForDepression() {
  const TStr InFNm = "depression/FebJun09-w5mfq10.QtBs.rar";
  const int MinWrdLen = 7;
  const int MinMemeFq = 10;
  const int ClustMeth = 1;
  const TStr Pref = TStr::Fmt("Dep23e-w%dfq%d", MinWrdLen, MinMemeFq);
  PQuoteBs QtBs;
  if (TZipIn::IsZipFNm(InFNm)) { TZipIn ZipIn(InFNm);  QtBs = TQuoteBs::Load(ZipIn); }
  else { TFIn FIn(InFNm);  QtBs = TQuoteBs::Load(FIn); }
  PClustNet Net = TClustNet::Load(TFIn("depression/FebJun09-w7fq10.ClustNet")); //TClustNet::GetFromQtBs(QtBs, MinMemeFq, MinWrdLen);
  printf("\n***Build done\n");
  TSnap::PrintInfo(Net);
  //{ TFOut FOut(Pref+".ClustNet"); Net->Save(FOut); }
  // cluster
  printf("\n***CLUST\n");
  Net = Net->GetSubGraph(7, 100, 10);
  TSnap::DelSelfEdges(Net);
  TIntPrV KeeepEdgeV;
  if (ClustMeth == 5) { Net->ClustGreedyTopDown(KeeepEdgeV); }
  else { Net->ClustKeepSingleEdge(ClustMeth, KeeepEdgeV); }

  // delete
  IAssert(Net->IsNode(2547905)); Net->DelNode(2547905);
  IAssert(Net->IsNode(2547910)); Net->DelNode(2547910);
  IAssert(Net->IsNode(2564208)); Net->DelNode(2564208);
  IAssert(Net->IsNode(55769)); Net->DelNode(55769);
  QtBs->DelQtId(2547905);
  QtBs->DelQtId(2547910);
  QtBs->DelQtId(2564208);
  QtBs->DelQtId(55769);
  //*/


  Net->MakeClusters(KeeepEdgeV);
  TVec<TIntV> ClustV;
  Net->GetClusters(ClustV);
  QtBs->CreateClusters(ClustV); // set clusters
  //{ TFOut FOut(Pref+"-clust.QtBs"); QtBs->Save(FOut); }
  // save clusters
  TIntV QtIdV;
  QtBs->GetQtIdVByFq(QtIdV, MinWrdLen, 50, false); QtIdV.Del(100, QtIdV.Len()-1);
  QtBs->SaveQuotes(QtIdV, Pref+"-quotes.txt");
  //QtBs->SaveClusters(Pref+"a", true);
  //QtBs->SaveClusters(Pref+"b", false);
  const TUrlTy CntUrlTy = utUndef;
  TSecTm MinTm(2009,2,1), MaxTm(2009,7,31); //QtBs->GetMinMaxTm(MinTm, MaxTm);
  //QtBs->GetQtIdVByFq(QtIdV, MinWrdLen, MinMemeFq, true, "", "", CntUrlTy);
  TStrV StrQtIdV; TStr("789996,1413261,236762,599878,788250,1342267,1048831,1110510,1394318,1283009,1426713,1394397,12677,788690,957195,795390,788689,877586,404516,2111129,1406451,1734005").SplitOnAllCh(',', StrQtIdV);
  QtIdV.Clr(); for (int i = 0; i < StrQtIdV.Len(); i++)  { QtIdV.Add(StrQtIdV[i].GetInt()); }
  QtBs->SaveClusters(QtIdV, Pref+"c", false);
  QtBs->SaveForFlash(QtIdV, Pref+"-top10", tmu4Hour, 10, MinTm, MaxTm, CntUrlTy);
  QtBs->SaveForFlash(QtIdV, Pref+"-top20", tmu4Hour, 20, MinTm, MaxTm, CntUrlTy);
  QtBs->SaveForFlash(QtIdV, Pref+"-top30", tmu4Hour, 30, MinTm, MaxTm, CntUrlTy);
  //QtBs->SaveForFlash(QtIdV, Pref+"-top50", tmu4Hour, 50, MinTm, MaxTm, CntUrlTy);
  //QtBs->SaveForFlash(QtIdV, Pref+"-top100", tmu4Hour, 100, MinTm, MaxTm, CntUrlTy);
}